{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "solving pendulum using actor-critic model\n",
    "\"\"\"\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Dropout, Input, GRU\n",
    "from tensorflow.keras.layers import Add, Concatenate\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import random\n",
    "from collections import deque\n",
    "\n",
    "def stack_samples(samples):\n",
    "\t\n",
    "\t\n",
    "\tobs_seqs = [train_data[0] for train_data in samples]\n",
    "\tactions = [train_data[1] for train_data in samples]\n",
    "\trewards = [train_data[2] for train_data in samples]\n",
    "\tnew_states = [train_data[3] for train_data in samples]\n",
    "\tdones = [train_data[4] for train_data in samples]\n",
    "\t\n",
    "\treturn current_states, actions, rewards, new_states, dones\n",
    "\t\n",
    "\n",
    "# determines how to assign values to each state, i.e. takes the state\n",
    "# and action (two-input model) and determines the corresponding value\n",
    "class ActorCritic:\n",
    "\tdef __init__(self, env, sess):\n",
    "\t\tself.env  = env\n",
    "\t\tself.sess = sess\n",
    "\n",
    "\t\tself.learning_rate = 0.0001\n",
    "\t\tself.epsilon = .9\n",
    "\t\tself.epsilon_decay = .99995\n",
    "\t\tself.gamma = .90\n",
    "\t\tself.tau   = .01\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                               Actor Model                             #\n",
    "\t\t# Chain rule: find the gradient of chaging the actor network params in  #\n",
    "\t\t# getting closest to the final value network predictions, i.e. de/dA    #\n",
    "\t\t# Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.memory = deque(maxlen=4000)\n",
    "\t\tself.actor_state_input, self.actor_action_input, self.actor_model = self.create_actor_model()\n",
    "\t\t_, self.target_actor_model = self.create_actor_model()\n",
    "\n",
    "\t\tself.actor_critic_grad = tf.placeholder(tf.float32,\n",
    "\t\t\t[None, self.env.action_space.shape[0]]) # where we will feed de/dC (from critic)\n",
    "\n",
    "\t\tactor_model_weights = self.actor_model.trainable_weights\n",
    "\t\tself.actor_grads = tf.gradients(self.actor_model.output,\n",
    "\t\t\tactor_model_weights, -self.actor_critic_grad) # dC/dA (from actor)\n",
    "\t\tgrads = zip(self.actor_grads, actor_model_weights)\n",
    "\t\tself.optimize = tf.train.AdamOptimizer(self.learning_rate).apply_gradients(grads)\n",
    "\n",
    "\t\t# ===================================================================== #\n",
    "\t\t#                              Critic Model                             #\n",
    "\t\t# ===================================================================== #\n",
    "\n",
    "\t\tself.critic_state_input, self.critic_action_input, \\\n",
    "\t\t\tself.critic_model = self.create_critic_model()\n",
    "\t\t_, _, self.target_critic_model = self.create_critic_model()\n",
    "\n",
    "\t\tself.critic_grads = tf.gradients(self.critic_model.output,\n",
    "\t\t\tself.critic_action_input) # where we calcaulte de/dC for feeding above\n",
    "\n",
    "\t\t# Initialize for later gradient calculations\n",
    "\t\tself.sess.run(tf.initialize_all_variables())\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Definitions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef create_actor_model(self):\n",
    "\t\tstate_input = Input(shape=(None,self.env.observation_space.shape[0]))\n",
    "\t\th1 = Dense(500, activation='relu')(state_input)\n",
    "\t\tactor_rnn,state_h = GRU(256, return_state=True)(h1)\n",
    "        h2 = Dense(500, activation='relu')(state_h)\n",
    "\t\toutput = Dense(self.env.action_space.shape[0], activation='tanh')(h3)\n",
    "\n",
    "\t\tmodel = Model([state_input], output)\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, model\n",
    "\n",
    "\tdef create_critic_model(self):\n",
    "\t\tstate_input = Input(shape=self.env.observation_space.shape)\n",
    "\t\tstate_h1 = Dense(500, activation='relu')(state_input)\n",
    "\t\tcritic_rnn,state_h2 = GRU(256, return_state=True)(state_h1)\n",
    "\n",
    "\t\taction_input = Input(shape=self.env.action_space.shape)\n",
    "\t\taction_h1    = Dense(500)(action_input)\n",
    "\n",
    "\t\tmerged    = Concatenate()([state_h2, action_h1])\n",
    "\t\tmerged_h1 = Dense(500, activation='relu')(merged)\n",
    "\t\toutput = Dense(1, activation='linear')(merged_h1)\n",
    "\t\tmodel  = Model([state_input,action_input],output)\n",
    "\n",
    "\t\tadam  = Adam(lr=0.0001)\n",
    "\t\tmodel.compile(loss=\"mse\", optimizer=adam)\n",
    "\t\treturn state_input, action_input, model\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                               Model Training                              #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef remember(self, cur_state, action, reward, new_state, done):\n",
    "\t\tself.memory.append([cur_state, action, reward, new_state, done])\n",
    "\n",
    "\tdef _train_actor(self, samples):\n",
    "\t\t\n",
    "\t\t\tobs_seqs, actions, rewards, new_seqs, _ =  stack_samples(samples)\n",
    "\t\t\tpredicted_actions = self.actor_model.predict(cur_states)\n",
    "\t\t\tgrads = self.sess.run(self.critic_grads, feed_dict={\n",
    "\t\t\t\tself.critic_state_input:  cur_states,\n",
    "\t\t\t\tself.critic_action_input: predicted_actions\n",
    "\t\t\t})[0]\n",
    "\n",
    "\t\t\tself.sess.run(self.optimize, feed_dict={\n",
    "\t\t\t\tself.actor_state_input: cur_states,\n",
    "\t\t\t\tself.actor_critic_grad: grads\n",
    "\t\t\t})\n",
    "\n",
    "\tdef _train_critic(self, samples):\n",
    "   \n",
    "\n",
    "\t\tcur_states, actions, rewards, new_states, dones = stack_samples(samples)\n",
    "\t\ttarget_actions = self.target_actor_model.predict(new_states)\n",
    "\t\tfuture_rewards = self.target_critic_model.predict([new_states, target_actions])\n",
    "\t\t\n",
    "\t\trewards += self.gamma * future_rewards * (1 - dones)\n",
    "\t\t\n",
    "\t\tevaluation = self.critic_model.fit([cur_states, actions], rewards, verbose=0)\n",
    "\t\t#print(evaluation.history)\n",
    "\tdef train(self):\n",
    "\t\tbatch_size = 256\n",
    "\t\tif len(self.memory) < batch_size:\n",
    "\t\t\treturn\n",
    "\n",
    "\t\trewards = []\n",
    "\t\tsamples = random.sample(self.memory, batch_size)\n",
    "\t\tself.samples = samples\n",
    "\t\tself._train_critic(samples)\n",
    "\t\tself._train_actor(samples)\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                         Target Model Updating                             #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef _update_actor_target(self):\n",
    "\t\tactor_model_weights  = self.actor_model.get_weights()\n",
    "\t\tactor_target_weights = self.target_actor_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(actor_target_weights)):\n",
    "\t\t\tactor_target_weights[i] = actor_model_weights[i]*self.tau + actor_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_actor_model.set_weights(actor_target_weights)\n",
    "\n",
    "\tdef _update_critic_target(self):\n",
    "\t\tcritic_model_weights  = self.critic_model.get_weights()\n",
    "\t\tcritic_target_weights = self.target_critic_model.get_weights()\n",
    "\t\t\n",
    "\t\tfor i in range(len(critic_target_weights)):\n",
    "\t\t\tcritic_target_weights[i] = critic_model_weights[i]*self.tau + critic_target_weights[i]*(1-self.tau)\n",
    "\t\tself.target_critic_model.set_weights(critic_target_weights)\n",
    "\n",
    "\tdef update_target(self):\n",
    "\t\tself._update_actor_target()\n",
    "\t\tself._update_critic_target()\n",
    "\n",
    "\t# ========================================================================= #\n",
    "\t#                              Model Predictions                            #\n",
    "\t# ========================================================================= #\n",
    "\n",
    "\tdef act(self, cur_state):\n",
    "\t\tself.epsilon *= self.epsilon_decay\n",
    "\t\tif np.random.random() < self.epsilon:\n",
    "\t\t\treturn self.actor_model.predict(cur_state)*2 + np.random.normal()\n",
    "\t\treturn self.actor_model.predict(cur_state)*2\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_should_use.py:193: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "trial:0\n",
      "[-1.4709074]\n",
      "trial:1\n",
      "[-19.382574]\n",
      "trial:2\n",
      "[-17.520435]\n",
      "trial:3\n",
      "[-15.306483]\n",
      "trial:4\n",
      "[-14.2388935]\n",
      "trial:5\n",
      "[-18.63116]\n",
      "trial:6\n",
      "[-15.7308445]\n",
      "trial:7\n",
      "[-15.7315855]\n",
      "trial:8\n",
      "[-15.403826]\n",
      "trial:9\n",
      "[-17.029757]\n",
      "trial:10\n",
      "[-16.03441]\n",
      "trial:11\n",
      "[-10.72435]\n",
      "trial:12\n",
      "[-14.768001]\n",
      "trial:13\n",
      "[-8.619002]\n",
      "trial:14\n",
      "[-10.166697]\n",
      "trial:15\n",
      "[-17.756784]\n",
      "trial:16\n",
      "[-17.452412]\n",
      "trial:17\n",
      "[-14.498742]\n",
      "trial:18\n",
      "[-11.080182]\n",
      "trial:19\n",
      "[-18.604654]\n",
      "trial:20\n",
      "[-9.36736]\n",
      "trial:21\n",
      "[-8.334008]\n",
      "trial:22\n",
      "[-4.514587]\n",
      "trial:23\n",
      "[-12.912566]\n",
      "trial:24\n",
      "[-23.22045]\n",
      "trial:25\n",
      "[-6.888916]\n",
      "trial:26\n",
      "[-8.733273]\n",
      "trial:27\n",
      "[-18.022373]\n",
      "trial:28\n",
      "[-5.145549]\n",
      "trial:29\n",
      "[-0.07214142]\n",
      "trial:30\n",
      "[-13.076322]\n",
      "trial:31\n",
      "[-8.61939]\n",
      "trial:32\n",
      "[-1.0955737]\n",
      "trial:33\n",
      "[-22.135962]\n",
      "trial:34\n",
      "[-0.22295392]\n",
      "trial:35\n",
      "[-6.1236043]\n",
      "trial:36\n",
      "[-17.286156]\n",
      "trial:37\n",
      "[-0.10973699]\n",
      "trial:38\n",
      "[-8.224843]\n",
      "trial:39\n",
      "[-2.0568125]\n",
      "trial:40\n",
      "[-0.06211594]\n",
      "trial:41\n",
      "[-4.0351915]\n",
      "trial:42\n",
      "[-0.05135815]\n",
      "trial:43\n",
      "[-0.00404448]\n",
      "trial:44\n",
      "[-8.318268]\n",
      "trial:45\n",
      "[-0.0233779]\n",
      "trial:46\n",
      "[-0.00185184]\n",
      "trial:47\n",
      "[-0.01446036]\n",
      "trial:48\n",
      "[-0.00320845]\n",
      "trial:49\n",
      "[-0.03889276]\n",
      "trial:50\n",
      "[-0.00958339]\n",
      "trial:51\n",
      "[-0.01270416]\n",
      "trial:52\n",
      "[-0.01883486]\n",
      "trial:53\n",
      "[-0.01600553]\n",
      "trial:54\n",
      "[-0.01439999]\n",
      "trial:55\n",
      "[-0.01567666]\n",
      "trial:56\n",
      "[-0.01811929]\n",
      "trial:57\n",
      "[-0.00433024]\n",
      "trial:58\n",
      "[-0.02078888]\n",
      "trial:59\n",
      "[-0.01608613]\n",
      "trial:60\n",
      "[-0.01894901]\n",
      "trial:61\n",
      "[-0.01915195]\n",
      "trial:62\n",
      "[-0.02542907]\n",
      "trial:63\n",
      "[-0.01205522]\n",
      "trial:64\n",
      "[-0.01674628]\n",
      "trial:65\n",
      "[-0.02674664]\n",
      "trial:66\n",
      "[-0.0155308]\n",
      "trial:67\n",
      "[-0.02103985]\n",
      "trial:68\n",
      "[-0.02221483]\n",
      "trial:69\n",
      "[-0.01899401]\n",
      "trial:70\n",
      "[-0.01931999]\n",
      "trial:71\n",
      "[-0.00678201]\n",
      "trial:72\n",
      "[-0.0178212]\n",
      "trial:73\n",
      "[-0.01838512]\n",
      "trial:74\n",
      "[-0.02622633]\n",
      "trial:75\n",
      "[-0.02904913]\n",
      "trial:76\n",
      "[-0.01756252]\n",
      "trial:77\n",
      "[-5.798752]\n",
      "trial:78\n",
      "[-0.01632133]\n",
      "trial:79\n",
      "[-0.0144076]\n",
      "trial:80\n",
      "[-0.01634561]\n",
      "trial:81\n",
      "[-2.1386175]\n",
      "trial:82\n",
      "[-0.01282954]\n",
      "trial:83\n",
      "[-0.01694369]\n",
      "trial:84\n",
      "[-0.00703546]\n",
      "trial:85\n",
      "[-0.02353159]\n",
      "trial:86\n",
      "[-0.01379559]\n",
      "trial:87\n",
      "[-0.00784179]\n",
      "trial:88\n",
      "[-0.01158375]\n",
      "trial:89\n",
      "[-0.01262122]\n",
      "trial:90\n",
      "[-0.01710485]\n",
      "trial:91\n",
      "[-0.01498491]\n",
      "trial:92\n",
      "[-0.01496802]\n",
      "trial:93\n",
      "[-0.01497802]\n",
      "trial:94\n",
      "[-0.01683223]\n",
      "trial:95\n",
      "[-0.00530027]\n",
      "trial:96\n",
      "[-0.01520001]\n",
      "trial:97\n",
      "[-0.01291954]\n",
      "trial:98\n",
      "[-0.01919345]\n",
      "trial:99\n",
      "[-0.00504622]\n",
      "trial:100\n",
      "[-0.00870032]\n",
      "trial:101\n",
      "[-0.0166171]\n",
      "trial:102\n",
      "[-0.01483979]\n",
      "trial:103\n",
      "[-0.01576297]\n",
      "trial:104\n",
      "[-0.01615123]\n",
      "trial:105\n",
      "[-0.01586954]\n",
      "trial:106\n",
      "[-0.01337176]\n",
      "trial:107\n",
      "[-0.01900725]\n",
      "trial:108\n",
      "[-0.00569406]\n",
      "trial:109\n",
      "[-0.01658658]\n",
      "trial:110\n",
      "[-0.01676113]\n",
      "trial:111\n",
      "[-0.01533814]\n",
      "trial:112\n",
      "[-0.01341606]\n",
      "trial:113\n",
      "[-0.02017977]\n",
      "trial:114\n",
      "[-0.01343245]\n",
      "trial:115\n",
      "[-0.01900205]\n",
      "trial:116\n",
      "[-0.00923794]\n",
      "trial:117\n",
      "[-0.00574253]\n",
      "trial:118\n",
      "[-0.01310444]\n",
      "trial:119\n",
      "[-0.01827688]\n",
      "trial:120\n",
      "[-0.01508313]\n",
      "trial:121\n",
      "[-0.00714861]\n",
      "trial:122\n",
      "[-0.01916804]\n",
      "trial:123\n",
      "[-0.01202975]\n",
      "trial:124\n",
      "[-0.02002115]\n",
      "trial:125\n",
      "[-0.01524988]\n",
      "trial:126\n",
      "[-0.0084835]\n",
      "trial:127\n",
      "[-0.02022774]\n",
      "trial:128\n",
      "[-0.01359174]\n",
      "trial:129\n",
      "[-0.00922045]\n",
      "trial:130\n",
      "[-0.02037779]\n",
      "trial:131\n",
      "[-0.01872296]\n",
      "trial:132\n",
      "[-0.01398168]\n",
      "trial:133\n",
      "[-0.0179784]\n",
      "trial:134\n",
      "[-0.01823119]\n",
      "trial:135\n",
      "[-0.01597575]\n",
      "trial:136\n",
      "[-0.00907116]\n",
      "trial:137\n",
      "[-0.01761812]\n",
      "trial:138\n",
      "[-0.01319274]\n",
      "trial:139\n",
      "[-0.01273214]\n",
      "trial:140\n",
      "[-0.01383581]\n",
      "trial:141\n",
      "[-0.0154989]\n",
      "trial:142\n",
      "[-0.0213029]\n",
      "trial:143\n",
      "[-0.02270907]\n",
      "trial:144\n",
      "[-0.01242783]\n",
      "trial:145\n",
      "[-0.02025035]\n",
      "trial:146\n",
      "[-0.02137059]\n",
      "trial:147\n",
      "[-0.0121228]\n",
      "trial:148\n",
      "[-0.02697988]\n",
      "trial:149\n",
      "[-0.02041909]\n",
      "trial:150\n",
      "[-0.01194584]\n",
      "trial:151\n",
      "[-0.02108318]\n",
      "trial:152\n",
      "[-0.00780763]\n",
      "trial:153\n",
      "[-0.01436973]\n",
      "trial:154\n",
      "[-0.01066801]\n",
      "trial:155\n",
      "[-0.01405584]\n",
      "trial:156\n",
      "[-0.02242532]\n",
      "trial:157\n",
      "[-0.01241341]\n",
      "trial:158\n",
      "[-0.01900714]\n",
      "trial:159\n",
      "[-0.0110427]\n",
      "trial:160\n",
      "[-0.01717351]\n",
      "trial:161\n",
      "[-0.01925875]\n",
      "trial:162\n",
      "[-0.01328584]\n",
      "trial:163\n",
      "[-0.02323352]\n",
      "trial:164\n",
      "[-0.013258]\n",
      "trial:165\n",
      "[-0.0107688]\n",
      "trial:166\n",
      "[-0.01935307]\n",
      "trial:167\n",
      "[-0.0240659]\n",
      "trial:168\n",
      "[-0.01783168]\n",
      "trial:169\n",
      "[-0.01896972]\n",
      "trial:170\n",
      "[-0.02205075]\n",
      "trial:171\n",
      "[-0.02467752]\n",
      "trial:172\n",
      "[-0.02407362]\n",
      "trial:173\n",
      "[-0.01723439]\n",
      "trial:174\n",
      "[-0.02013997]\n",
      "trial:175\n",
      "[-0.02369069]\n",
      "trial:176\n",
      "[-0.02136697]\n",
      "trial:177\n",
      "[-0.01901076]\n",
      "trial:178\n",
      "[-0.02523747]\n",
      "trial:179\n",
      "[-0.01233395]\n",
      "trial:180\n",
      "[-0.01641912]\n",
      "trial:181\n",
      "[-0.01216715]\n",
      "trial:182\n",
      "[-0.02573084]\n",
      "trial:183\n",
      "[-0.03130922]\n",
      "trial:184\n",
      "[-0.01247637]\n",
      "trial:185\n",
      "[-0.02423921]\n",
      "trial:186\n",
      "[-0.02168536]\n",
      "trial:187\n",
      "[-0.02079885]\n",
      "trial:188\n",
      "[-0.02666252]\n",
      "trial:189\n",
      "[-0.01407196]\n",
      "trial:190\n",
      "[-0.02141044]\n",
      "trial:191\n",
      "[-0.01251335]\n",
      "trial:192\n",
      "[-0.01250621]\n",
      "trial:193\n",
      "[-0.02187206]\n",
      "trial:194\n",
      "[-0.02440181]\n",
      "trial:195\n",
      "[-0.01455239]\n",
      "trial:196\n",
      "[-0.02403909]\n",
      "trial:197\n",
      "[-0.02452361]\n",
      "trial:198\n",
      "[-0.02338326]\n",
      "trial:199\n",
      "[-0.02210737]\n",
      "trial:200\n",
      "[-0.02196637]\n",
      "trial:201\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-924e6c052d18>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-11-924e6c052d18>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m                                 \u001b[0mactor_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m                         \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-8a8fe26b0ec0>\u001b[0m in \u001b[0;36mupdate_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    173\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mupdate_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_actor_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_critic_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0;31m# ========================================================================= #\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-8a8fe26b0ec0>\u001b[0m in \u001b[0;36m_update_critic_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    165\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_update_critic_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m                 \u001b[0mcritic_model_weights\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m                 \u001b[0mcritic_target_weights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_critic_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcritic_target_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mget_weights\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    151\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_distribution_strategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mby_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36mget_weights\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1128\u001b[0m     \"\"\"\n\u001b[1;32m   1129\u001b[0m     \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1130\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_get_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mget_updates_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36mbatch_get_value\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m   3008\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Cannot get value inside Tensorflow graph function.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3009\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3010\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3011\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3012\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1156\u001b[0m     \u001b[0;31m# Create a fetch handler to take care of the structure of fetches.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1157\u001b[0m     fetch_handler = _FetchHandler(\n\u001b[0;32m-> 1158\u001b[0;31m         self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)\n\u001b[0m\u001b[1;32m   1159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1160\u001b[0m     \u001b[0;31m# Run request and get response.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, graph, fetches, feeds, feed_handles)\u001b[0m\n\u001b[1;32m    472\u001b[0m     \"\"\"\n\u001b[1;32m    473\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 474\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    475\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_targets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m    262\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m       \u001b[0;31m# NOTE(touts): This is also the code path for namedtuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    265\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches)\u001b[0m\n\u001b[1;32m    371\u001b[0m     \"\"\"\n\u001b[1;32m    372\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    374\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    371\u001b[0m     \"\"\"\n\u001b[1;32m    372\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    374\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def main():\n",
    "\tsess = tf.Session()\n",
    "\tK.set_session(sess)\n",
    "\tenv = gym.make(\"Pendulum-v0\")\n",
    "\tactor_critic = ActorCritic(env, sess)\n",
    "\n",
    "\tnum_trials = 10000\n",
    "\ttrial_len  = 200\n",
    "\n",
    "\tfor i in range(num_trials):\n",
    "\t\tprint(\"trial:\" + str(i))\n",
    "\t\tcur_state = env.reset()\n",
    "\t\taction = env.action_space.sample()\n",
    "\t\treward_sum = 0\n",
    "        obs_list = []        \n",
    "        obs_list.append(cur_state)\n",
    "\t\tfor j in range(trial_len):\n",
    "\t\t\t#env.render()           \n",
    "\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\taction = actor_critic.act(cur_state)\n",
    "\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\treward += reward\n",
    "\t\t\tif j == (trial_len - 1):\n",
    "\t\t\t\tdone = True\n",
    "\t\t\t\tprint(reward)\n",
    "\n",
    "\t\t\tif (j % 5 == 0):\n",
    "\t\t\t\tactor_critic.train()\n",
    "\t\t\t\tactor_critic.update_target()   \n",
    "\t\t\t\n",
    "\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "            obs_seq = np.asarray(obs_list)\n",
    "            obs_list.append(new_state)\n",
    "            next_obs_seq = np.asarray(obs_list)\n",
    "\t\t\tactor_critic.remember(obs_seq, action, reward, next_obs_seq, done)\n",
    "\t\t\tcur_state = new_state\n",
    "\n",
    "\t\tif (i % 5 == 0):\n",
    "\t\t\tcur_state = env.reset()\n",
    "\t\t\tfor j in range(500):\n",
    "\t\t\t\tenv.render()\n",
    "\t\t\t\tcur_state = cur_state.reshape((1, env.observation_space.shape[0]))\n",
    "\t\t\t\taction = actor_critic.act(cur_state)\n",
    "\t\t\t\taction = action.reshape((1, env.action_space.shape[0]))\n",
    "\n",
    "\t\t\t\tnew_state, reward, done, _ = env.step(action)\n",
    "\t\t\t\t#reward += reward\n",
    "\t\t\t\t#if j == (trial_len - 1):\n",
    "\t\t\t\t\t#done = True\n",
    "\t\t\t\t\t#print(reward)\n",
    "\n",
    "\t\t\t\t#if (j % 5 == 0):\n",
    "\t\t\t\t#    actor_critic.train()\n",
    "\t\t\t\t#    actor_critic.update_target()   \n",
    "\t\t\t\t\n",
    "\t\t\t\tnew_state = new_state.reshape((1, env.observation_space.shape[0]))\n",
    "\n",
    "\t\t\t\t#actor_critic.remember(cur_state, action, reward, new_state, done)\n",
    "\t\t\t\tcur_state = new_state\n",
    "\t\t\t\t\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
